#! /bin/bash
# shili.lzy@taobao.com, zhangxun@taobao.com
#
# HOW to start:
#   nohup $PWD/proc-monitor.sh --daemon </dev/null >/dev/null 2>&1 &
#

cd `dirname $0`/.
cfgfile=../conf/proc-monitor.conf
pidfile=../var/proc-monitor.pid
logfile_prefix_time=../logs/time/proc-monitor_time.log
logfile_prefix_mem=../logs/mem/proc-monitor_mem.log
logfile_prefix_net=../logs/net/proc-monitor_net.log
logfile_prefix_cpu=../logs/cpu/proc-monitor_cpu.log
prefix="/-----------------"
error_time=0
running_flag=0
time_flag=0
mem_flag=0
net_flag=0
cpu_flag=0
time_filter_conf=../conf/time_filter.conf
mem_filter_conf=../conf/mem_filter.conf
net_filter_conf=../conf/net_filter.conf
cpu_filter_conf=../conf/cpu_filter.conf
time_tmp=/tmp/proc-monitor.time
mem_tmp=/tmp/proc-monitor.mem
net_tmp=/tmp/proc-monitor.net
cpu_tmp=/tmp/proc-monitor.cpu

function count_prefix()
{
    local stack_num=$1
    prefix=${prefix:1:stack_num}
    return 0
}


function print_proc_tree
{
    local pid=$1
    local stacknum=$2
    prefix="/-----------------"

    ((stacknum=stacknum-2))                                 
    [ $stacknum -eq 0 ] && return   ## protection
    [ -d /proc/$pid ] || return
    ppid=`cat /proc/$pid/status | grep ^PPid | awk '{print $2}'`
    cmdline=`cat /proc/$pid/cmdline | tr '\0' ' '`
    count_prefix $stacknum
    echo >&5 "ERROR $check_item:$prefix pid($pid) ppid($ppid) $cmdline"
    print_proc_tree $ppid $stacknum $ckeck_item
}

function time_filter()
{
    local pid_in=$1
    local user=$2
    time_flag=0
    local cmd=`cat /proc/$pid_in/cmdline`
    cat $time_filter_conf | grep -v '#' >/tmp/proc_time_filter.gg
    while read user_filter cmd_filter;do
        if [ "$user_filter" == '*' ];then
            if [ "$cmd_filter" == '*' ];then
                time_flag=1
                #echo >&5 "filter point1 user filter=* & cmd=*"
                break
            elif [[ "$cmd" =~ "$cmd_filter" ]];then
                time_flag=1
                #echo >&5 "filter point2 user filter=* & cmd==cmdfilter"
                break
            else
                #echo >&5 "filter point3 user filter=* cmdfilter=$cmd_filter,cmd=$cmd"
                :
            fi
        else
            if [ "$user" == "$user_filter" ];then
                if [[ "$cmd" =~ "$cmd_filter" ]];then
                    #echo >&5 "filter point4 user=userfilter=$user, cmd==cmdfilter=$cmd_filter"
                    time_flag=1
                    break
                elif [ "$cmd_filter" == '*' ];then
                    #echo >&5 "filter point5 user=userfilter=$user,cmd filter=*"
                    time_flag=1
                    break 
                else
                    #echo >&5 "filter point6 user=userfilter=$user,cmdfilter=$cmd_filter,cmd=$cmd"
                    :
                fi
            else
                #echo >&5 "filter point7 user !=userfilter !=* user=$user ,userfilter=$user_filter"
                :
            fi
        fi
    done < /tmp/proc_time_filter.gg
    rm -f /tmp/tmp/proc_time_filter.gg

    #specail for login shell
    if [ "$cmd" == "bash" ];then
        time_flag=1
    elif [ "$cmd" == "-bash" ];then
        time_flag=1
    else
        :
    fi

    return 0
}

function mem_filter()
{
    local pid_in=$1
    local user=$2
    mem_flag=0
    local cmd=`cat /proc/$pid_in/cmdline`
    cat $mem_filter_conf | grep -v '#' >/tmp/proc_mem_filter.gg
    while read user_filter cmd_filter;do
        if [ "$user_filter" == '*' ];then
            if [ "$cmd_filter" == '*' ];then
                mem_flag=1
                #echo >&5 "filter point1 user filter=* & cmd=*"
                break
            elif [[ "$cmd" =~ "$cmd_filter" ]];then
                mem_flag=1
                #echo >&5 "filter point2 user filter=* & cmd==cmdfilter"
                break
            else
                #echo >&5 "filter point3 user filter=* cmdfilter=$cmd_filter,cmd=$cmd"
                :
            fi
        else
            if [ "$user" == "$user_filter" ];then
                if [[ "$cmd" =~ "$cmd_filter" ]];then
                    #echo >&5 "filter point4 user=userfilter=$user, cmd==cmdfilter=$cmd_filter"
                    mem_flag=1
                    break
                elif [ "$cmd_filter" == '*' ];then
                    #echo >&5 "filter point5 user=userfilter=$user,cmd filter=*"
                    mem_flag=1
                    break 
                else
                    #echo >&5 "filter point6 user=userfilter=$user,cmdfilter=$cmd_filter,cmd=$cmd"
                    :
                fi
            else
                #echo >&5 "filter point7 user !=userfilter !=* user=$user ,userfilter=$user_filter"
                :
            fi
        fi
    done < /tmp/proc_mem_filter.gg
    rm -f /tmp/tmp/proc_mem_filter.gg

    return 0
}

function cpu_filter()
{
    local pid_in=$1
    local user=$2
    cpu_flag=0
    local cmd=`cat /proc/$pid_in/cmdline`
    cat $cpu_filter_conf | grep -v '#' >/tmp/proc_cpu_filter.gg
    while read user_filter cmd_filter;do
        if [ "$user_filter" == '*' ];then
            if [ "$cmd_filter" == '*' ];then
                cpu_flag=1
                #echo >&5 "filter point1 user filter=* & cmd=*"
                break
            elif [[ "$cmd" =~ "$cmd_filter" ]];then
                cpu_flag=1
                #echo >&5 "filter point2 user filter=* & cmd==cmdfilter"
                break
            else
                #echo >&5 "filter point3 user filter=* cmdfilter=$cmd_filter,cmd=$cmd"
                :
            fi
        else
            if [ "$user" == "$user_filter" ];then
                if [[ "$cmd" =~ "$cmd_filter" ]];then
                    #echo >&5 "filter point4 user=userfilter=$user, cmd==cmdfilter=$cmd_filter"
                    cpu_flag=1
                    break
                elif [ "$cmd_filter" == '*' ];then
                    #echo >&5 "filter point5 user=userfilter=$user,cmd filter=*"
                    cpu_flag=1
                    break 
                else
                    #echo >&5 "filter point6 user=userfilter=$user,cmdfilter=$cmd_filter,cmd=$cmd"
                    :
                fi
            else
                #echo >&5 "filter point7 user !=userfilter !=* user=$user ,userfilter=$user_filter"
                :
            fi
        fi
    done < /tmp/proc_cpu_filter.gg
    rm -f /tmp/tmp/proc_cpu_filter.gg

    return 0
}

function net_filter()
{
    local pid_in=$1
    local user=$2
    net_flag=0
    local cmd=`cat /proc/$pid_in/cmdline`
    cat $net_filter_conf | grep -v '#' >/tmp/proc_net_filter.gg
    while read user_filter cmd_filter;do
        if [ "$user_filter" == '*' ];then
            if [ "$cmd_filter" == '*' ];then
                net_flag=1
                #echo >&5 "filter point1 user filter=* & cmd=*"
                break
            elif [[ "$cmd" =~ "$cmd_filter" ]];then
                net_flag=1
                #echo >&5 "filter point2 user filter=* & cmd==cmdfilter"
                break
            else
                #echo >&5 "filter point3 user filter=* cmdfilter=$cmd_filter,cmd=$cmd"
                :
            fi
        else
            if [ "$user" == "$user_filter" ];then
                if [[ "$cmd" =~ "$cmd_filter" ]];then
                    #echo >&5 "filter point4 user=userfilter=$user, cmd==cmdfilter=$cmd_filter"
                    net_flag=1
                    break
                elif [ "$cmd_filter" == '*' ];then
                    #echo >&5 "filter point5 user=userfilter=$user,cmd filter=*"
                    net_flag=1
                    break 
                else
                    #echo >&5 "filter point6 user=userfilter=$user,cmdfilter=$cmd_filter,cmd=$cmd"
                    :
                fi
            else
                #echo >&5 "filter point7 user !=userfilter !=* user=$user ,userfilter=$user_filter"
                :
            fi
        fi
    done < /tmp/proc_net_filter.gg
    rm -f /tmp/tmp/proc_net_filter.gg

    return 0
}

function time_diff()
{
    start_time=`echo $1 | grep :`
    end_time=`echo $2 | grep :`
    if [ -z $start_time ];then
        return 0
    fi
    if [ -z $end_time ];then
        return 0
    fi

    start_s=`date -d "$start_time" +%s`
    end_s=`date -d "$end_time" +%s`
    ((diff=$end_s - $start_s))
    if [ $diff -gt $min_time_threshold ];then
        error_time=$diff
        return 1    
    fi
    return 0
}

function check_proc_mem
{
    logfile_mem=$logfile_prefix_mem.`date +%d`
    check_item="MEM"
    [ "$logfile_mem" != "$lastlogfile_mem" ] && exec 5>$logfile_mem
    [ "$logfile_mem" == "$lastlogfile_mem" ] && exec 5>>$logfile_mem
    lastlogfile_mem=$logfile_mem
    
    echo >&5 "`date +%F\ %T`| checking (min_mem_threshold = $min_mem_threshold MByte) ..."
    ps --no-headers ax -o pid,ppid,user,vsz,rsz,%cpu,time,cmd |
    while read pid ppid user vsz rsz cpu_percent time cmd; do
        ((rsz = rsz / 1024))   ## convert to MByte
        if (($rsz > $min_mem_threshold)); then
            mem_filter $pid $user
            if [ $mem_flag -eq 1 ];then
                :
            else
                echo >&5 "ERROR $check_item:Found pid($pid), rsz($rsz/$min_mem_threshold MByte)"
                print_proc_tree $pid 20 $check_item
            fi
        fi
    done
}

function check_proc_netstat
{
    logfile_net=$logfile_prefix_net.`date +%d`
    check_item="NETSTAT"
    [ "$logfile_net" != "$lastlogfile_net" ] && exec 5>$logfile_net
    [ "$logfile_net" == "$lastlogfile_net" ] && exec 5>>$logfile_net
    lastlogfile_net=$logfile_net
    
    echo >&5 "`date +%F\ %T`| checking (min_net_threshold = $min_net_threshold links) ..."
    netstat -anpt |grep -v "-" | awk '{print $7}'|uniq -c |sort -nr | head -10 | awk -F/ '{print $1}'|
    while read num pid; do
        if (($num > $min_net_threshold)); then
            user=`ps aux | grep "$pid" | grep -v grep | awk '{print $1}'`
            net_filter $pid  $user
            if [ $net_flag -eq 1 ];then
                :
            else
                echo >&5 "ERROR $check_item:Found pid($pid),($num/$min_net_threshold links)"
                print_proc_tree $pid 20 $check_item
            fi
        fi
    done
}

function check_proc_time
{
    logfile_time=$logfile_prefix_time.`date +%d`
    check_item="TIME"
    [ "$logfile_time" != "$lastlogfile_time" ] && exec 5>$logfile_time
    [ "$logfile_time" == "$lastlogfile_time" ] && exec 5>>$logfile_time
    lastlogfile_time=$logfile_time
    
    echo >&5 "`date +%F\ %T`| checking (min_time_threshold = $min_time_threshold seconds) ..."
    ps --no-headers ax -o pid,ppid,user,stime,%cpu,time,cmd |
    while read pid ppid user stime cpu_percent time cmd; do
        time_now=`date +%H:%M`
        error_time=0
        time_diff "$stime" "$time_now"
        if [ $? -ne 0 ]; then
            time_filter $pid $user
            if [ $time_flag -eq 1 ];then
                :
            else
                echo >&5 "ERROR $check_item:Found pid($pid), time($error_time/$min_time_threshold seconds)"
                print_proc_tree $pid 20 $check_item
            fi
        fi
    done
}

function check_proc_cpu
{
    logfile_cpu=$logfile_prefix_cpu.`date +%d`
    check_item="TIME"
    [ "$logfile_time" != "$lastlogfile_time" ] && exec 5>$logfile_cpu
    [ "$logfile_time" == "$lastlogfile_time" ] && exec 5>>$logfile_cpu
    lastlogfile_time=$logfile_cpu
    
    echo >&5 "`date +%F\ %T`| checking (min_cpu_threshold = $min_cpu_threshold %) ..."
    ps --no-headers ax -o pid,ppid,user,stime,%cpu,time,cmd |
    while read pid ppid user stime cpu_percent time cmd; do
        if [ $cpu_percent -gt $min_cpu_threshold ]; then
            cpu_filter $pid $user
            if [ $cpu_flag -eq 1 ];then
                :
            else
                echo >&5 "ERROR $check_item:Found pid($pid), cpu($cpu_percent/$min_cpu_threshold %)"
                print_proc_tree $pid 20 $check_item
            fi
        fi
    done
}
################################################################

if [ "$1" != "--daemon" ]; then
    echo >&2 "Process monitor. Usage: $0 --daemon"
    echo >&2 "HOW to start me,"
    echo >&2
    echo >&2 "nohup $0 --daemon </dev/null >/dev/null 2>&1 &"
    echo >&2
    echo >&2 "How to kill me!"
    echo >&2 "kill -INT -$PID,donot forget the '-' before PID"
    echo >&2
    exit 1
fi

trap "rm -f $pidfile;echo "INT";exit 0" INT
trap "rm -f $pidfile;echo "TERM";exit 0" TERM
#check_self_running $0
[ -f $pidfile ] && exit 0
echo $$ >$pidfile

lastlogfile_time=$logfile_prefix_time.`date +%d`      ## required
lastlogfile_mem=$logfile_prefix_mem.`date +%d`      ## required
lastlogfile_net=$logfile_prefix_net.`date +%d`      ## required
lastlogfile_cpu=$logfile_prefix_cpu.`date +%d`      ## required
source $cfgfile

while [ 1 ]; do
    [ -f $cfgfile ] && source $cfgfile
    check_proc_mem
    check_proc_time
    check_proc_netstat
    check_proc_cpu
    sleep 120
done

# vim:ts=4:sw=4:ai:et